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0 /* Copyright Aventail Corporation 1997-2000; All Rights Reserved */ 

\ 11 T*' 0 iS the SSL environ ^ent file; it defines functions used by 

1 the SSL module as callbacks for managing mutexes, memory, I/O 

3 and user interaction. */ 

4 ' 

5 #include "sslmain.h" 

6 finclude "sslldap.h" 

7 #include "ldapcert.h" 

8 #include <aglobal.h> 

9 #include <bsafe.h> 
10 #include <pkcs.h> 

. <due to the size of this file and the small portion of it which is relevant here, we include only the single function SSLEncodO 
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int FAR EXPORT SSLEncode (SSPacket. *ibuf, SSPacket *obuf , int flag, void 'handle) 

SSSSLHandle *ref = handle; 
SSSSLFlowConnection *conh =■ & <ref->conn) ; 
SSLContext *ctx = ref->ctx; 
uint32 ilen; 

uint32 len = ibuf ? ibuf->len : 0; 
int ssloppy =0; 

#ifdef HYPER_DEBUG 

int i; 
#endif 

SSLErr err> 

uint32 wrtp =0; 

if (flag & S5 STATEDUMP) 
{ 

SSLBuffer block; 
unsigned totalSize; 

PSSLStateDump dump *= (PSSLStateDump) obuf->data; 

if (obuf->len < 4096) 
{ 

obuf->len = 4096; 

return ENCODE_BUFFER_TOO_SMALL; 

) 

// get the SSL state 
// 

if ((err » SSLExportContext (ctx, fiblock) ) != SSLNoErr) 

if (Global Update) 

GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_ERROR, 

1DS_SSL_EXP0RTC0NTEXTFAILED, err) ; 
•Lcuurii *" x. f ^ 

> 

// compute the total size of the data 

v totalSize = block. length + -sizeof ( S SLS tat e Dump) ; 

// validate the output buffer size 

if (obuf->len < totalSize) 
{ 

obuf->len = totalSize; 
SSLFreeBufferUblock, &ctx->sysCtx) ; 
return ENCODE BUFFER TOO SMALL; 
) ~ - ' 

// put the SSLStateDump structure at the beginning of the output buffer 

dump->SSLContext = ctx; 
dump->ContextSize = sizeof (SSLContext) ; 
dump->sSLState.data « (uint8 *) (dump+1) ; 



* i ) ' ) 
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1025 dump->SSLState. length «=» block. length; 

1026 

1027 //. copy the. SSL state to the output buffer 

1028 // 

1029 memcpy(dump->sSLSt ate. data, block. data, block. length ) ; 

1030 obuf->len « totalSize; 

1031 SSLFreeBuffer(&block, &ctx->sysCtx) ; 
1032 

1033 if (GlobalUpdate) 

1034 GlobalUpdate (sslLogHandle, S5_LCK5J4ISC, S5_LOG_VERBOSE, 

™H IDS_SSL_EXPORTEDCONTEXT, obuf->len) / 

1036 • 

1037 return 0; 

1038 } 
1039 

1040 if (ref->endtime > 0) 

1041 if (time( (time_t ■*) NULL) >=* ref->endtime) { 

1042 if (GlobalUpdate) 

GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 

. IDS_SSL_LIFETIMEEXCEEDED) / 



1043 
1044 



1045 return -1; 

1046 ) 
1047 

1048 SSLGetWritePendingSize(ctx, &wrtp) ; 

1049 #ifdef HYPER_DEBUG 

1050 if(wrtp) 

1051 if (GlobalUpdate) 

1°? 2 GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 

* I D S_S S L_B YTES PEN D I NGWRI TE , wrtp) / 

1054 #endif 

1055 

1056 if (flag & S5J3ATAGRAM) 

1057 if (ref->ssloppy) 

1058 { 

1° 59 if((rt = SSLSetSloppyMode(ctx, 1)) != SSLNoErr) 

1060 { 

1° 61 if (GlobalUpdate) 

J!?" GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_WARNING, 

IDS_SSL S S LOP PYMODE FAILED, rt) / 
1° 64 return -1; ~ 

1065 ) 

1066 s sloppy =1; 

1067 > 

1068 else 

1069 { 

1070 /* UDP naked, baby! */ 

1071 #ifdef HYPER_DEBUG 

1072 if (GlobalUpdate) 

GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 
1075 #endif I DS_S S L_GOT DATAGRAM , flag, ibuf->len) ; 

1076 

1077 #ifdef OPTIMIZE_UDP_NAKED 

l^H /* this is m ^ch cleaner and faster, but causes inconsistency in the 

1079 API from the caller. Sigh. */ 

1080 *obuf - *ibuf; 

1081 #else 

1082 #ifdef WIN32 

}®l 3 A n obuf->data = HeapAlloc(GetProcessHeap() , 0, ibuf->len) ; 

1084 #else 

1° 85 obuf->data = malloc.(ibuf->len) ; 

1086 #endif 

1087 if <obuf->data NULL) {' 

1088 #ifdef HYPERJDEBUG 

1089 FPRlNTFOtderr, _T ("Returning error, buf data is null in 
obufdata\n") ) ; 

1090 #endif 

1° 9 1 return SSLMemoryErr; 

1092 } 

1° 93 memcpy{obuf->data, ibuf->data, ibuf->len); 
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109 4 obuf->len = ibuf->len; 

1095 fendif 

1096 return ibuf->len; 

1097 } 
1098 

1099 
1100- 

1101 if (flag &.S5 ENCODE) { 
1102 

1103 #ifdef HYPER_DEBUG 
.1104 if (GlobalOpdate) 

JJjJ;* GlobalUpdate (sslLogHandle, S5 LOG_MISC, S5_LOG_VERBOSE, 

. . IDS_SSL ENCODINGB YTES , ibuf->len) ; 

1107 #ifndef AUTOSOCKS . 

for(i = 0; i < ibuf->len; i++) 

1109 FPRINTF(stderr, _T("%02x "), ibuf->data [ij ) ; 

1110 FPRINTF(stderr, T("\n")>; 
.1111 #endif " 

1112 #endif 



if(ibuf->len > SSL_MAX_ENCODE_SIZE) { 

conn->modctx. log. update (sslLogHandle, iS5_LOG_MISC,S5_LOG_VERBOSE / 

IDS_SSL_MAXENCODESIZEEXCEEDED, 

iw ^ , ibuf->len, SSL MAX ENCODE SIZE) ; 

ibuf->len « SSL MAX ENCODE SIZE; ~ " , 

) " " 

if (obuf->data != NULL) { 

if(obuf->len < (int) (ibuf->len + SSL_HEADLEN + 64 + wrtp)) { 

conn->modctx . log.update (sslLogHandle, S5_LOG_MISC, S5_LOG_DEBUG, 
en IDS_SSL_BUFFERTOOSHORT, 

wrtp) j . (ibuf->len + SSLJiEADLEN 

obuf->len « (int) ibuf->len + SSL_HEADLEN + 64 + wrtp; 
if(ssloppy) SSLS'etSloppyModefctx, 0); 
return ENCODE BUFFER TOO SMALL; 
) ~ ~ 

conn->writebuffer.data = obuf->data; 
conn->writebuffer;len = obuf->len; 
conn->writebuffer.off » SSL_HEADLEN; 
COnn->writeflag = SSL_FLOW_WRITE NOMAKEBUF; 
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) else 



conn->writeflag « 0; 



ilen = (uint32) ibuf->len; 

if ((err = SSLWrite (ibuf->data, &ilen, ctx))) { 

conn->modctx . log.update (sslLogHandle, S5_LOG_MISC, S5_LOG_ERROR, 

IDS SSL WRITEERROR, err) ; 
if(ssloppy) SSLSetSloppyMode(ctx, 0); - - 

return -1; 

} 

if (conn->writebuffer.off > OxFFFF) { 

conn->modctx . log . update { sslLogHandle , S5_LOG_MISC, S5_LOG_ERROR, 

IDS_SSL_PACKETTOOBIG, 
... . t „ conn->writebuffer.off); 
if^ssloppy) SSLSetSloppyMode(ctx, 0); 
return -1; 

) 

if (obuf->data !=NULL) { 

/* Here we shift the semantics of writebuffer; off now points 
to the beginning of the data, and len points to the end of 
the data, not the length of the buffer, which we no longer 
need to know. since we don't be depositing anything new in it */ 
obuf->len = conn->writebuf fer .of f ; 
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conn->writebuf f er . len 
conn->writebuf f er . off 



1163 • 
1164 

1165 ) else { 

1166 
1167 
1168 
1169 
1170 

1171 else { 

1172 #ifndef JflNDOWS 
1173 

1174 #else 

1175 #ifdef WIN32 
1176 
1177 

SSL_HEADLEN); 

1178 #endif 

1179 #end±f 
1180 
1181 
1182 

S5_LO(?J3RROR, IDS_SSL_MALLOCFAILED) 



conn->writebuf f er . off; 
SSL HEADLEN; 



if (conn->writebuffer.off =» SSL_HEADLEN) 

/* Wow! Some thoughtful soul in SSLFlowWrite has left us a 4 
byte offset in the writebuf fer so we can insert our header 
without needing to mallop a new buffer and meracpy into it! */ 
obuf->data = conn->writebuf fer. data; 



obuf->data = malloc ( conn- >writebuf fer .len + SSL_HEADLEN) / 
obuf->data 



HeapAlloc(GetProcessHeap<) , 0, 

conn->writebuffer.len + 



if (obuf->data = NULL) { 

conn- >modctx.. log. update (sslLogHandle, S5_LOG_MISC, 



1183 
1184 
1185 
1186 
1187 
1188 
1189 

1190 #ifdef WIN32 
1191 

1192 #else 
1193 

1194 #endif 

1195 

1196 

1197 

1198 

1199 } 
1200 
1201 
1202 
1203 
1204 
1205 
1206 
1207 
1208 
1209 
1210 
1211 
1212 



> 



if(ssloppy) SSLSetSloppyMode(ctx, 0), 
return -1; 



memcpy(obuf->data + SSL_HEADLEN, 

conn->writebuf fer. data + conn->writebuffer .off , 
conn->writebuffer.len - conn->writebuf fer. of f ) ; 

HeapFree(GetProcessHeap<), 0, conn->writebuf fer . data) ; 

free (conn->writebuf fer. data) ; 



} 



obuf->len = (int) (conn->writebuf fer . len - 

conn->writebuffer.off + SSL_HEADLEN) ; 



Obuf->data[0] » S S L_HEAD VERS I ON / 
obuf->data[l] = conn->state; 

obuf->data[2] = (uint8) ( (conn->writebuf fer. len - conn->writebuf fer, of f ) 

» 8); 

obuf->data[3] ** (uint8) ( (conn->writebuf fer. len - conn->writebuf fer .of f j 

& OxFF) ; 

conn->writebuf fer .data » NULL/ 
conn->writebuffer.len =0; 
conn->writebuf f er . of f = 0; 
if (GlobalUpdate) 

GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 

1213 #if de£ HYPER_DEBUG IDS_SSL_ENCODERETURn£nG/ obuf->le„, lien, ; 

1214 #ifndef AUTOSOCKS 

1215 for(i «» 0; i < obuf->len; i++) 

1216 FPRlNTF(stderr, _T("%02x n ) f obuf->data[i] ) ; 

1217 FPRINTF(stderr, T( n \n")); 

1218 #endif 

1219 #endif 
1220 
1221 

1222 ) 
1223 

1224 /* we must be decoding, instead of encoding.. */ 

1225 #ifdef HYPER_DEBUG 

1226 if (GlobalUpdate) 

1227 GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 

IDS_SSL DECODINGBYTES , ibuf->len) ; 

1229 if (GlobalUpdate) 

1230 GlobalUpdate (sslLogHandle, S5_LOGJ4ISC, S5_LOG_VERBOSE, 

1231 IDS_SSL_READBUFFERCOMINGIN, 



if(ssloppy) SSLSetSloppyMode(ctx, 0); 
return (int) ilen; 
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1232. conn->readbuffer .len - conn->readbuf fer.of f) ; 

1233 #ifndef AUTOSOCKS 

1234 for<i = 0/ i < ibuf->len; i++) 

1235 FPRINTF(stderr, _T("%02x ibuf->data[i] ) ; 

1236 FPRINTF(stderr, _T( n \n n )); 

1237 #endif 

1238 #endif 

1239 . 

1240 if (ibuf->len < SSL_HEADLEN) 

1241 { 

1242 conn->modctx. log. update (sslLogHandle, S5_LOG_MISC, S5_LOG_ERROR, 

1243 IDS_SSL_DECODEINCOMPLETEPACKET); 

1244 if(ssloppy) SSLSetSloppyMode (ctx, 0); 

1245 return -1; 

1246 } 
1247 

1248 if (ibuf->data[0] ! = SSL_HEADVERSION) { 

1249 conn->modctx. log. update (sslLogHandle, S5_LOG_MISC,S5_LOG_ERROR, 

12 50 I DS_S SL_HEADERVERS IONMI SMATCH , 

12 51 S SL_HEADVERS I ON , ibuf->data [0] ) ; 
.1252 if(ssloppy) SSLSetSloppyMode (ctx, 0); 

1253 return -1; 

1254 > 

1255 if (ibuf->datatl) 1= conn->state) { 

1256 conn->modctx. log. update (sslLogHandle, S5_LOG_MISC, S5_LOG_ERROR, 

1257 * I DS_SSL_HEADERSTATEMI SMATCH , 

1258 conn->state, ibuf->data [1) ) ; 

1259 if(ssloppy) SSLSetSloppyMode (ctx, 0); 

1260 return -1; 

1261 } 
1262 

1263 ilen = ((uint8) ibuf->data[2] ) « 8; 

1264 ilen |- (uint8) ibuf->data[3] ; - 

1265 ilen SSLJiEADLEN; /* we must include the header in the length because 

1266 no man is an ilen. Er, because it's the length 

1267 of the whole record, including the header. */ 
1268 

1269 #ifdef HYPERJDEBUG 

1270 if (GlobalUpdate) 

1271 GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 

1272 IDS SSL PACKETSIZE, ilen) ; 

1273 #endif " " 
1274 

1275 if(ibuf->len < (int) (ilen)) { 

I 27 6 conn->modctx . log .update (sslLogHandle, S5_LOG_MISC, S5_LOG_ERROR, 

1211 I DS_S S L_DEC ODEI NCOM P L ETE P AC KET ) / . 

1278 if(ssloppy) SSLSetSloppyMode (ctx, 0); 

1279 return -1; 

1280 > 
1281 

1282 #if 0 

1283 if(ibuf->len > (int) (ilen)) { 

1284 conn->modctx . log . update (sslLogHandle, S5_LOG_MISC, S5_LOG_ERR0R, 

1285 IDS_SSL_DECODEOVERFULLPACKET) / 

1286 if(ssloppy) SSLSetSloppyMode (ctx, 0) ; 

1287 return -1; 

1288 } 

1289 #endif 
1290 

1291 SSLGetReadPendingSize(ctx, swrtp) ; /* this should be zero, but seems to 

1 292 not always be, so we be safe.. */ 
1293 

1294 #if 0 

1295 /* we need to choose the size of the obuf here; since SSL adds some 

1296 . boundary information the size of the input should be big enough. 

1297 if SSL+ starts to support compression this assumption will have 

1298 to change. */ 

1299 len « conn->readbuf fer .len - conn->readbuf fer.of f + wrtp + ilen; 

1300 #else 

1301 /* OK, so we decided to change it. Now we Jcnow the record can't 

1302 be larger than 16K. */ 



File: socksS / common / modules / authentication / ssl / sslenyx Page 6 of 7 

Revision 1.136.2.1, by marcvh 

1303 ./* len *■ conn->readbuffer.len - conn->readbuf fer.of f + wrtp + 16384; */ 

1304 /* Hmra. . the above seems to cause telnet to studder, while a fixed 32k 

1305 size fixes it; so 32k it shall be... */ 

1306 len » 32767; 

1307 #endif 

1308 if (obuf->data l~ NULL) { 

1309 if(obuf->len < (int) len) { 

1310 if (GlobalUpdate) 

1311 GlobalUpdate(sslLogHandle, S5_LOG_MISC, S5_LOG_DEBUG, 

1312 IDS_SSL_BUFFERTOOSMALL, obuf->len, len) ; 

1313 obuf->len « (int) len; 

1314 if(ssloppy) SSLSetSloppyMode (ctx, 0); 

1315 return ENC0DE_BUFFER_TOO SMALL; 

1316 } 

1317 } else { 

1318 fifndef ^WINDOWS 

1319 obuf->data = (unsigned char *) malloc(len); 

1320 #else 

1321 #ifdef WIN32 

1322 obuf->data » HeapAlloc (GetProcessHeap( ) , 0, len); 

1323 #endif 

1324 #endif 

1325 ) 

1326 #if 0 

1327 /* must read all the data we can.. */ 

1328 len « ilen + conn->readbuf fer.len; 

1329 #endif 
1330 

1331 if (conn->readbuffer.data = NULL) { 

1332 /* conn->readbuf fer.data « malloc ( (size_t) (len - SSL_HEADLEN) ) ; */ 

1 333 /* Try to re-use the input buffer instead of needing to create 

1334 a new one and memcpy into it. readflag lets us know we did 

1335 this so we don't try to change or free the space later */ 

1336 conn->readbuffer.data = ibuf->data; 

1337 conn->readbuffer.len = ilen; 

1338 conn->readbuffer.off = SSL_HEADLEN; 

1339 COnn->readflag = SSL_FLOW_READ_NOOWNBUF; 

1340 } else- { 

1341 conn->readbuffer.data » realloc (conn->readbuf fer.data, 

1 3 42 (size_t) (len - SSL_HEADLEN) ) ; 

1343 conn->readflag =0; 

1344 if <conn->readbuf fer.data « NULL) { 

1345 conn->modctx . log . update ( sslLogHandle , S5_L0G_MISC, S5_LOG_ERROR, 

1346 IDS_SSL_REALLOC FAILED) ; 

1347 if(ssloppy) SSLSetSloppyMode (ctx, 0) ; 

1348 return -1; 
134? } 

1350 memcpy (conn->readbuf fer.data + conn->readbuf fer .len, 

1351 ibuf->data + SSL_HEADLEN, <size_t) (ilen - SSL_HEADLEN) ) ; 

1352 conn->readbuffer.len += ilen - SSL HEADLEN; 

1353 } " 
1354 

1355 if ((err = SSLRead( (void *) obuf->data, &len, ctx)) && 

1356 (err !« SSLWouldBlockErr) ) { 

1357 conn->modctx . log . update (sslLogHandle, S5_LOG_MISC, SS^LOG^ERROR, 

1358 I DS_S SL_READERROR, err); 

1359 if(ssloppy) SSLSetSloppyMode (ctx; 0) ; 

1360 return -1; 

1361 } 

1362 obuf->len *= (int) len; 

1363 #ifdef HYPER_DEBUG 

1364 if (GlobalUpdate) 

1365 GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_VERBOSE, 

1366 IDS_SSL_ENCODERETURNINGBYTES, 

1367 ilen, len); 

1368 if (GlobalUpdate) 

1369 GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_L0G_VERB0SE, 

1370 IDS_SSL_READBUFFERGOINGOUT, 

1371' conn->readbuffer,len - conn»>readbuffer .of f ) ; 

1372 iifndef AUTOSOCKS . 

1373 for(i = 0; i < len; i++) 
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1374 FPRINTF(stderr, _T("%02x ") , obuf->data [i] ) ; 

1375 FPRINTF(stderr, T("\n H )); 

1376 #endif 

1377 #endlf 

1378 if (conn->readflag & SSL_FLOW_READ_NOOWNBUF) 

1379 if (conn->readbuffer.off < conn->readbuffer.Xen) { 

1380 BYTE *t; 
1381 

1382 if (GlobalUpdate) 

!383 GlobalUpdate (sslLogHandle, S5_LOG_MISC, S5_LOG_WARNING / 

1384 IDS_SSL_ENCODELEAVINGDATA, 

conn->readbuffer.len - conn->readbuffer.off) ; 
1386 t - malloc{conn->readbuffer.len - conn->readbuffer.off ) / 

.1387 memcpy(t, conn->readbuf fer .data + conn->readbuffer.of f , 

13 .88 conn->readbuffer.len - conn->readbuf fer .of f ) ; 

1389 conn->readbuf fer. data = t; 

1390 conn->readbuffer.len « conn->readbuffer.len - conn->readbuf fer .of f ; 

1391 conn->readbuffer.off = 0/ 

1392 } else { 

1393 conn->readbuf fer. data = NULL; 

1394 conn->readbuffer.off » 0; . 

1395 conn->readbuf fer.len =0; 

1396 ) 

1397 if(ssloppy) SSLSetSloppyMode (ctx, 0); 

1398 return (int) ileh; 

1399 > 



